#!/usr/bin/python
#
# Copyright 2007 DeWitt Clinton, All Rights Reserved

"""Read and parse URLs and return information about HTML link relationships."""

__author__ = 'dewitt@unto.net'

import getopt
import html5lib
import httplib2
import lxml.etree as etree
import pprint
import sys
import urlparse
from html5lib.treebuilders import getTreeBuilder


def GetRelationships(url, from_id=None, cache_dir='.cache'):
  """Read a url and return a sequence of identity tuples.

  Identity tuples are of the form:

     (str(from_id), str(to_id), list(relationships))

  Args:
    url: the URL to be loaded and parsed
    from_id: the identity URL the relationship should be considered from
    cache_dir: the directory on disk to cache HTTP responses
  Returns:
    a sequence of identity tuples in the form (from_id, to_id, [relationship])
  """
  if not from_id:
    from_id = url

  # TODO(dewitt): Make a HEAD request first, only GET when needed
  h = httplib2.Http(cache_dir)
  resp, content = h.request(url)
  p = html5lib.HTMLParser(tree=getTreeBuilder('etree', etree))
  tree = p.parse(content)
  base = GetBase(resp, tree)
  links = GetLinks(tree, base)
  return [(from_id, to_id, relationships) for to_id, relationships in links]


def GetBase(resp, tree):
  """Determine the base URL for links from an HTML parse tree and headers.

  Args:
    resp: a httplib2 response
    tree: an html5lib etree parse tree
  Returns:
    the base URL for outbound links
  """
  try:
    base = tree.find('.//base').attrib['href']
  except AttributeError:
    base = resp['content-location']
  return base


def GetLinks(tree, base):
  """Get all outbound links and their rel values from a HTML tree.

  Args:
    tree: an html5lib etree parse tree
    base: the base URL for outbound links
  Returns:
    a list of (href, [rel values]) tuples
  """
  links = []
  for node in tree.findall('.//link') + tree.findall('.//a'):
    rel_values = GetRelValues(node)
    href = node.attrib['href']
    absolute_href = MakeAbsolute(href, base)
    links.append((absolute_href, rel_values))
  return links


def MakeAbsolute(url, base):
  """Convert relative links to absolute links.

  Args:
    url: a relative or absolute url
    base: the base URL to be used to make relative urls absolute
  Returns:
    an absolute URL
  """
  scheme, netloc, path, params, query, fragment = urlparse.urlparse(url)
  if not netloc:
    return urlparse.urljoin(base, url)
  else:
    return url

def GetRelValues(node):
  """Parse a node and return the sequence of associated rel values.

  Args:
    node: an ElementTree node
  Returns:
    a sequence of space-delimited rel values
  """
  try:
    rel = node.attrib['rel']
    return rel.split()
  except:
    return []


def Usage():
  print """Usage: relationship_parser.py [-h --help] [-f from_id] url

Read the identity URL and return a python tuple string expressing the
relationships.

Args:
  -h --help   Print this help
  -f --from   The 'from' identity URL, overrides the url being parsed
"""


def Main(argv=None):
  if argv is None:
    argv = sys.argv
  try:
    opts, args = getopt.getopt(argv[1:], "hf:", ["help", "from"])
  except getopt.GetoptError, msg:
    Usage()
    return 2
  from_id = None
  for o, a in opts:
    if o in ("-h", "--help"):
      Usage()
      return 2
    if o in ("-f", "--from"):
      from_id = a
  try:
    url = args[0]
  except:
    Usage()
    return 2
  relationships = GetRelationships(url, from_id)
  pprint.pprint(relationships)
  return 0

if __name__ == '__main__':
  sys.exit(Main())
